{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoded IDs: [11, 12, 5, 11, 14, 18, 8, 15, 10, 19, 14, 6]\n",
      "Encoded: ['h', 'i', ',', 'h', 'o', 'w', 'a', 'r', 'e', 'y', 'o', '?']\n",
      "Decoded: h i , h o w a r e y o ?\n"
     ]
    }
   ],
   "source": [
    "# from tokenizers import Tokenizer, models, pre_tokenizers, trainers\n",
    "# from datasets import load_dataset\n",
    "\n",
    "# # 加载数据集，这里使用一个简单的示例数据集\n",
    "# dataset = load_dataset('wikitext', 'wikitext-103-raw-v1')\n",
    "\n",
    "# # 将数据合并为一个文本列表\n",
    "# corpus = [line for line in dataset['train']['text']]\n",
    "\n",
    "# # 创建一个 BPE 模型\n",
    "# tokenizer = Tokenizer(models.BPE())\n",
    "\n",
    "# # 预处理：空格分词和标点符号分开\n",
    "# tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()\n",
    "\n",
    "# # 使用 BPE 算法进行训练\n",
    "# trainer = trainers.BpeTrainer(special_tokens=[\"[PAD]\", \"[UNK]\", \"[BOS]\", \"[EOS]\"])\n",
    "# tokenizer.train_from_iterator(corpus, trainer=trainer)\n",
    "\n",
    "# # 保存训练好的 tokenizer\n",
    "# tokenizer.save(\"custom_tokenizer.json\")\n",
    "\n",
    "# 加载训练好的 tokenizer\n",
    "from tokenizers import Tokenizer\n",
    "tokenizer = Tokenizer.from_file(\"custom_tokenizer.json\")\n",
    "\n",
    "# 进行编码和解码测试\n",
    "encoded = tokenizer.encode(\"hi, how are you?\")\n",
    "print(\"Encoded IDs:\", encoded.ids)\n",
    "print(\"Encoded:\", encoded.tokens)\n",
    "decoded = tokenizer.decode(encoded.ids)\n",
    "print(\"Decoded:\", decoded)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
